#coding:utf-8
#根据mysql中的词典对未标注的文档进行初步标定
import os
import sys
import shutil
project_path=os.getcwd()
gather_path=os.path.abspath(os.path.dirname(project_path)+os.path.sep+".")
sys.path.append(gather_path)
from bs4 import BeautifulSoup
father_path=os.path.abspath(os.path.dirname(project_path)+os.path.sep+"..")
print ("project path :"+project_path)
print ("project father path:"+father_path)
file_type = '.txt'

data_root_path = father_path+"/abc_project_data/unmark_html/"
data_output_path = father_path+"/abc_project_data/analysed_html/"

if os.path.isdir(data_output_path):
    shutil.rmtree(data_output_path)
os.mkdir(data_output_path)

def get_file_name(data_root_path,file_type):
    file_name=[]
    for root ,dir, files in os.walk(data_root_path):
        for file in files :
            if file_type in file :
                file_name.append(file)
    return file_name
def main():
    #extract company people and industry in mysql db
    file_name_list = get_file_name(data_root_path, file_type)
    for file in file_name_list:
        input_html = open(data_root_path+file,'r').read().decode('utf-8')
        soup = BeautifulSoup(input_html, 'lxml')
        origin_text = soup.get_text().encode('utf-8')
        output_file = open(data_output_path + file, 'w')
        output_file.write(origin_text)

if __name__ == "__main__":
    main()